
package com.kinth.zgwk;

import java.io.File;
import java.util.*;

import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.slf4j.LoggerFactory;
import org.apache.log4j.xml.DOMConfigurator;
import org.slf4j.Logger;


import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.PriorityScheduler;

/**
 * 
 * <p>
 * Title: 获得总的列表ip.
 * </p>
 * <p>
 * 1. 一个新的公司入口页：http://qiye.99114.com/6884ewq
 * 然后分层提取，并给url设置优先级。这样，公司页面的将是最高优先级的。
 * 不过这还是需要一次性存储很多的url。不过经过测试应该可以。
 * 关于ip问题，解决方法就是时间延迟。自己无法控制request的请求时间，因为封装在框架里面。而且都是死有函数，无法覆写。所以
 * 就在添加request的时候添加延迟。每五秒一个。但是通过观察数据库上面的数据，还是一秒一个较多。应该是添加了之后一直
 * 累积在一起，然后一段时间一起爬取
 * 
 * 2. 上面的特点和中国制造网的分层抓取一样。但是多了一个图片识别的功能。在ImageDetection这个类里面。直接调用返回结果。
 *     其实可以一次返回几张图片的结果，但是自己觉得一张一张也比较方便。
 * 3. 根据目前的信息看，这里的数据质量不高，很多没有邮箱。有也是很差劲的，甚至有问题的。多半是新浪，网易的163,126，还有qq。
 * 4. 有的直接有qq。如果直接有qq，我就直接用qq拼接。不用读邮箱了。
 * 5. 总是报aliyun的socket问题。不知道为什么。
 * </p>
 * 
 * @author kaikai
 * @2017年4月1日
 */
public class MainForzgwk implements PageProcessor
{
	private Site site = Site.me().setUserAgent("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 "
			+ "(KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36").setRetryTimes(3).setSleepTime(100);

	//首页入口
	public static final String BIG_URL = "http://qiye.99114.com/";
	//列表页入口
	public  static final String BIG_URL2 = "http://qiye.99114.com/listing/[a-z]+_0_0_0_1_1.html";
	//公司列表页
	public  static final String BIG_URL3 = "http://qiye.99114.com/listing/[a-z]+_0_0_0_1_[0-9]+.html";
	//详情页
	public  static final String COMPANY_URL = "http://shop.99114.com/[0-9]+";
	//公司列表页最大分页数
	public static final int MAX_PAGE_NUM = 100;
	//每个公司列表页的公司链接数目
	public static final int PER_PAGE_NUM = 30;
	//添加每个链接的延迟，为了不被ip屏蔽
    public static final int SLEEP_TIME = 5000;
    
    protected Logger logger =  LoggerFactory.getLogger(getClass());
    
	@Override
	public void process(Page page)
	{
		if(page.getUrl().toString().equals(BIG_URL))
		{
			try
			{
				processBigUrl(page);
			} catch (Exception e)
			{
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}else if(page.getUrl().regex(BIG_URL2).match())
		{
			try
			{
				processBigUrl2(page);
			} catch (Exception e)
			{
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}else if(page.getUrl().regex(BIG_URL3).match())
		{
			try
			{
				processBigUrl3(page);
			} catch (Exception e)
			{
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}else if(page.getUrl().regex(COMPANY_URL).match())
		{
			try
			{
				getInfo(page);
			} catch (Exception e)
			{
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
	}

	//处理公司页面
	public void getInfo(Page page) throws Exception
	{

		logger.info("获取具体公司信息："+page.getUrl());

		String name = page.getHtml()
				.xpath("//body/div[@class='main']" + "//div[@class='con-box']//div[@class='comBorder']"
						+ "/div[@class='sj-line sjDiv1']/p[@class='companyname']/span"
						+ "/text()")
				.toString();

		String people = null;
		String address = null;
		String phone1 = null;
		String phone2 = null;
		String phone3 = null;
		String qq = null;
		String introduction = null;
		String product = null;

		String temp = null;
		//替换取出提取的字符串里面的html代码
		String regex;
		regex = "<[\\s\\S]*?>";
		
		if (name != null)
		{
			// 产品
			product = page.getHtml()
					.xpath("//body/div[@class='main']" + "//div[@class='con-box']//div[@class='comBorder']"
							+ "/div[@class='sj-line sjDiv2']/p[2]/span[2]"
							+ "/text()")
					.toString();
			
			qq = page.getHtml()
					.xpath("//body/div[@class='main']" + "//div[@class='con-box']//div[@class='comBorder']"
							+ "/div[@class='sj-line sjDiv1']/p[2]//a/@href"
							+ "")
					.toString();
			if(qq!=null)
			{
				String sTemp[] = qq.split("&");
				if(sTemp.length >= 2)
				{
					qq = sTemp[1];
					sTemp = qq.split("=");
					if(sTemp.length >= 2)
					{
						qq = sTemp[1];
					}
				}
			}
			if(qq != null)
			{
				qq = qq+"@qq.com";
			}
			
			//联系方式
			phone1 = page.getHtml()
					.xpath("//body/div[@class='main']" + "//div[@class='widget contactus  ']//div[@class='contactUl']"
							+ "//li[@class='picContact clearfix']/div[@class='contxt']/p[2]/span[1]"
							+ "/text()")
					.toString();
			
			temp = page.getHtml()
					.xpath("//body/div[@class='main']" + "//div[@class='widget contactus  ']//div[@class='contactUl']"
							+ "//li[@class='picContact clearfix']/div[@class='contxt']/p[2]/span[2]"
							+ "/img/@src")
					.toString();
			if(temp != null)
			{
				temp = ImageDetection.imageResult(temp);
			}
			phone1 = phone1+ temp;
			
			//联系人
			people = page.getHtml()
					.xpath("//body/div[@class='main']" + "//div[@class='widget contactus  ']//div[@class='contactUl']"
							+ "//li[@class='picContact clearfix']/div[@class='contxt']/p[1]/span[1]"
							+ "/text()")
					.toString();
			
			temp = page.getHtml()
					.xpath("//body/div[@class='main']" + "//div[@class='widget contactus  ']//div[@class='contactUl']"
							+ "//li[@class='picContact clearfix']/div[@class='contxt']/p[1]"
							+ "/text()")
					.toString();
			temp = temp.trim().replaceAll(" ", "");
			temp = temp.replaceAll("\n", "");
			people = people + " "+ temp;
			
			//使用循环
			int t = 1;
			while (t <= 4)
			{
				//电话
				temp = page.getHtml()
						.xpath("//body/div[@class='main']" + "//div[@class='widget contactus  ']//div[@class='contactUl']"
								+ "//li[@class='addIntroW']//div[@class='addIntroL']//li["+t+"]"
								+ "")
						.toString();

				temp = temp.replaceAll(regex, "").trim();
				temp = temp.replaceAll("&nbsp;", " ");
				
				if(temp!=null)
				{
					if(temp.substring(0, 1).equals("电"))
					{
						temp = page.getHtml()
								.xpath("//body/div[@class='main']" + "//div[@class='widget contactus  ']//div[@class='contactUl']"
										+ "//li[@class='addIntroW']//div[@class='addIntroL']//li["+t+"]"
										+ "/span[2]/img/@src")
								.toString();
						if(temp != null)
						{
							temp = ImageDetection.imageResult(temp);
						}
						phone2 = temp;
					    

					}
					else if( temp.substring(0, 1).equals("邮"))
					{

						temp = page.getHtml()
								.xpath("//body/div[@class='main']" + "//div[@class='widget contactus  ']//div[@class='contactUl']"
										+ "//li[@class='addIntroW']//div[@class='addIntroL']//li["+t+"]"
										+ "/span[2]/img/@src")
								.toString();
					
						if(temp != null)
						{
							temp = ImageDetection.imageResult(temp);
						}
						if(temp != null)
						{
							try{
								temp = getEmail(temp);
							}catch(Exception e)
							{
								e.printStackTrace();
								logger.error("邮箱格式化的时候有问题");
							}
						}
						qq = temp;
						
						qq = temp;
					}
					else if(temp.substring(0, 1).equals("传"))
					{
						temp = page.getHtml()
								.xpath("//body/div[@class='main']" + "//div[@class='widget contactus  ']//div[@class='contactUl']"
										+ "//li[@class='addIntroW']//div[@class='addIntroL']//li["+t+"]"
										+ "/span[2]/img/@src")
								.toString();
						if(temp != null)
						{
							temp = ImageDetection.imageResult(temp);
						}
						phone3 = temp;
					
					}else if(temp.substring(0, 1).equals("地"))
					{
						address = temp;
						
					}
				}
				t++;
			
		}
			
		}
		
		String urlNow = page.getUrl().toString();
		String industry = page.getRequest().getExtra("industry").toString();
		page.putField("name", name);
		page.putField("product", product);
		page.putField("address", address);
		page.putField("people", people);
		page.putField("qq", qq);
		page.putField("introduction", introduction);
		page.putField("phone1", phone1);
		page.putField("phone2", phone2);
		page.putField("phone3", phone3);
		page.putField("industry", industry);
		page.putField("url", urlNow);

	
	}
	
	//处理邮箱，因为通过图片读取的邮箱只能有数字和字母，没有字符。所以需要拆分
	public String getEmail(String email) throws Exception
	{
		String result = email;
		String sCom = email.substring(email.length()-3, email.length());
		if(sCom.equalsIgnoreCase("com"))
		{
			String sQQ = email.substring(email.length()-5,email.length()-3);
			if(sQQ.equalsIgnoreCase("qq"))
			{
				String qq = email.substring(0,email.length()-5);
				result = qq+"@qq.com";
			}else 
			{
				String s163 = email.substring(email.length()-6,email.length()-3);
				if(s163.equals("163")||s163.equals("126"))
				{
					String emailWangyi = email.substring(0,email.length()-6);
					result = emailWangyi +"@"+s163+".com";
				}else
				{
				int t = email.length()-4;
				for(int i = email.length()-4; i >= 0; i--)
				{
					Character c = email.charAt(i);
					if(!Character.isDigit(c))
					{
						t = i;
						break;
					}
				}
				String emailName = email.substring(0,t+1);
				String emailHost = email.substring(t+1,email.length()-3);
				result = emailName + "@"+emailHost +".com";
				}
			}
		}
		
		return result;
	}
	//处理列表页
	public void processBigUrl3(Page page) throws Exception
	{
		ArrayList<String> companyList = new ArrayList<String>();
		
		companyList = (ArrayList<String>) page.getHtml()
				.xpath("/html/body/div[@class='w1190 clearfix']" 
				        + "/div[@class='fl mt10 zuoce']/div[@class='cp-list']"
				        + "//a")
				.links().all();
		String industry = page.getRequest().getExtra("industry").toString();
		
		for (String url : companyList)
		{
			logger.info("得到公司链接并加入到request里面："+url);
			
			Request request = new Request(url).setPriority(2).putExtra("industry", industry);

			page.addTargetRequest(request); 
		}
	}
	//处理列表页首页
	public void processBigUrl2(Page page) throws Exception
	{
		String pageAmount = page.getHtml()
				.xpath("/html/body/div[@class='w1190 clearfix']" 
		        + "/div[@class='fl mt10 zuoce']/div[@class='crumb']/div[@class='fr']"
		        + "/span/text()")
				.toString();
        
		if (pageAmount != null)
		{
			String urlNow = page.getUrl().toString();
			int pageAmount2 = Integer.parseInt(pageAmount);
			int pageAmount3 = pageAmount2 / PER_PAGE_NUM + 1;

			if (pageAmount3 > MAX_PAGE_NUM)
			{
				pageAmount3 = MAX_PAGE_NUM;
			}
			
			String endHtml = urlNow.substring(urlNow.length()-5,urlNow.length());
			String startHtml = urlNow.substring(0,urlNow.length()-6);
			for (int i = 1; i <= pageAmount3; i++)
			{
				String urlnext = startHtml + i + endHtml;
				
				logger.info("得到公司列表的链接"+urlnext);
				Request request = new Request(urlnext).setPriority(1).putExtra("industry", page.getRequest().getExtra("industry"));
				page.addTargetRequest(request); 
			}
		}
	}
	//处理首页
	public void processBigUrl(Page page) throws Exception
	{
		ArrayList<String> nameList = new ArrayList<String>();
		ArrayList<String> urlList = new ArrayList<String>();
		
		urlList = (ArrayList<String>) page.getHtml()
				.xpath("//body/div[@class='Z-header-wrap']" + "/div[@class='Z-header']"
						+ "/div[@class='Z-nav-wrap J-Z-nav-wrap']//a").links().all();
		nameList = (ArrayList<String>) page.getHtml()
				.xpath("//body/div[@class='Z-header-wrap']" + "/div[@class='Z-header']"
						+ "/div[@class='Z-nav-wrap J-Z-nav-wrap']//a/text()").all();
		
		for (int i = 0; i < urlList.size(); i++)
		{
			Pattern p = Pattern.compile(BIG_URL2);
			Matcher m = p.matcher(urlList.get(i));
			if(m.find())
			{
				String url = urlList.get(i);
				String name = nameList.get(i);
				
				logger.info("得到公司列表链接的入口"+url);
				
				Request request = new Request(url).setPriority(0).putExtra("industry", name);
				
				page.addTargetRequest(request); 
			}
			
			
		}
	}
	@Override
	public Site getSite()
	{
		return site;
	}

	// 一个代替main函数的函数
	public static void runSpider()
	{
		System.out.println("爬取类别链接开始");
		MainForzgwk mainForzgwk = new MainForzgwk();
		Spider.create(mainForzgwk)
		.setScheduler(new PriorityScheduler())
		.addPipeline(new DataPipeline())//http://shop.99114.com/17024343
		.addUrl("http://qiye.99114.com/")
		.run();

	}

	public static void main(String[] args)
	{
		DOMConfigurator.configure(System.getProperty("user.dir")+File.separator+"log4j.xml");
		runSpider();
	}

	
}